#!/usr/bin/env python3
# cleaner.py - Copyright Matthew Leon Grinshpun, 2009
r"""cleaner.py - Usage: cleaner.py [options] args

cleaner.py presents a generalized interface for (very) simplified parsing
of text files through a sequentially applied list of regular expressions
and actions ("cleaners") culled from a JSON-formatted file (cleanregs.json 
by default).

Upon receiving a list of files passed in as an argument, cleaner.py
outputs the result of successive application of the cleaners to the file.

The json file format is as follows:
    -The file is one giant list bounded by brackets ([]).
    -Each cleaner is a dictionary entry, bounded by curly braces({}).
    -Cleaners must contain a "name" entry specifying their name.
    -Cleaners should contain a "comment" entry explaining their purpose.
    -Cleaners must contain a "regex" entry. Note that the JSON format forces
     us to escape our backslashes, so normal regex escapes should use a 
     double-backslash, as in Java (\\).
    -Cleaners may contain a "replacement" entry to designate replacements for
     matches. Again, backslashes must be escaped, ie ("\\1"). By default,
     matches are simply deleted (replaced by an empty string, "").
    -Cleaners may contain a "recursive" entry set to true or false. If set to
     true, the cleaner will be applied in a loop until it no longer matches. This
     setting is false by default.
    -Cleaners may contain a list of "flags", corresponding to the standard python
     regular expression flags. Long-form is preferred, ie (MULTILINE, DOTALL,
     IGNORECASE, LOCALE, UNICODE, VERBOSE). By default, no flags are specified.
    -Cleaners should contain a list of "tests". Each test is itself a list
     of only two elements, the first being a block of text, the second being
     the expected result of the application of the cleaner to this text. If
     cleaner.py is invoked with the --test or -t flag, these string tests
     will be applied to the cleaners.

TODO: Allow generalized unit tests for the entire series of cleaners.
TODO: Resolve certain utf-8 parsing issues (hopefully Python 3 will help). 
"""

import sys
import re
from optparse import OptionParser
from json import JSONDecoder

class Cleaner:
    """The regex based cleaner class."""
    def __init__(self, name, regex, replacement="", comment="", flags=0, recursive=False, tests=None):
        self.name = name
        self.comment = comment
        self.recursive = recursive
        self.tests = tests

        # private version of replacement must be masked when pickling
        self.replacement = replacement
        self._replacement = replacement if not replacement.startswith("lambda ") else eval(replacement)

        try:
            reflags = eval(" | ".join("re." + flag for flag in flags))
        except TypeError: # not iterable
            reflags = flags # we get here if this method is called with actual flags and not a string
        self.regex = re.compile(regex, reflags)

    def clean(self, string):
        cleaned = False 
        while not cleaned: # will loop for cleaners declared recursive
            string, subs = self.regex.subn(self._replacement, string)
            cleaned = True if subs == 0 or not self.recursive else False 
        return string

    def test(self, verbose=False):
        """Apply unit tests for the cleaner."""
        passed = True

        print("Test: " + self.name)
        for test in self.tests:
            dirty, desired = test[0], test[1]
            cleaned = self.clean(dirty)
            if verbose or desired != cleaned:
                print("Dirty string:" + dirty)
                print("Desired clean:" + desired)
                print("Got:" + cleaned)
            if desired == cleaned:
                print("PASSED")
            else:
                print("FAILED")
                passed = False
        print("")
        return passed

    def __getstate__(self):
        """Handle replacement (potentially a lambda func) during pickling"""
        dct = self.__dict__.copy()
        del dct["_replacement"] 
        return dct

    def __setstate__(self, state):
        """Deal with replacement when unpickling"""
        self.__dict__.update(state)
        self._replacement = self.replacement if not self.replacement.startswith("lambda ") else eval(self.replacement)

    def __str__(self):
        return str(self.__dict__)

def clean(string, cleaners):
    for cleaner in cleaners:
        string = cleaner.clean(string)
    return string

def import_cleaners(cleanerfile):
    importer = JSONDecoder(object_hook=lambda dct: Cleaner(**dct)) 
    with open(cleanerfile) as regfile:
        return importer.decode(regfile.read())

def test_cleaners(cleaners, verbose=False):
    passed_tests = True
    for cleaner in cleaners:
        passed_tests &= cleaner.test(verbose)
    print("All tests passed." if passed_tests else "Failed tests, check log.")

def clean_files(files, cleaners):
    for file in files:
        with open(file) as openfile:
            yield(clean(openfile.read(), cleaners))

def main():
    parser = OptionParser(usage="usage: %prog [options] args")
    parser.add_option("-r", "--regular-expression-file", 
                        dest="cleanerfile", default="cleanregs.json",
                        help="get regular expressions from JSON file (default: %default)")
    parser.add_option("-t", "--test-regular-expressions", 
                        action="store_true", dest="test", default=False,
                        help="run tests on regular epressions")
    parser.add_option("-v", "--verbose", 
                        action="store_true", dest="verbose", default=False,
                        help="verbose output when testing regular expressions")
    (options, args) = parser.parse_args()

    cleaners = import_cleaners(options.cleanerfile)

    if options.test:
        test_cleaners(cleaners, options.verbose)
    
    for cleanedfile in clean_files(args, cleaners):
        print(cleanedfile)

    return(0)

if __name__ == "__main__":
    sys.exit(main())
